clear all
clear matrix
version 17
capture log close
set matsize 11000, permanently
set more off, permanently

*-------------------------------------------------------------------------------
* Options	
*-------------------------------------------------------------------------------

global root =    // Define root: where did you save the bad-times folder

global main = "$root\bad-times\employers"

* Create folders & globals
cap mkdir "$main\prog"	    /* for do files */ 	
cap mkdir "$main\prog\wage_imputation"	/* for do-files for wage imputation */
cap mkdir "$main\graph"	    /* for graphs */ 
cap mkdir "$main\log"	    /* for log files */ 
cap mkdir "$main\temp"	    /* for temporary user generated data files */
cap mkdir "$main\results"	/* for final results */
cap mkdir "$main\data"		/* for additional data */
cap mkdir "$main\matlab"	/* for additional data */

global matlab     = "$main\matlab"
cap mkdir "$matlab\sconn_idfy"
cap mkdir "$matlab\functions"
cap mkdir "$matlab\data_new"
cap mkdir "$matlab\programs"

global prog       = "$main\prog"
global graph      = "$main\graph"
global log        = "$main\log"	
global temp       = "$main\temp"	
global results    = "$main\results"
global data       = "$main\data"
global imp		  = "$prog\02_wage_imputation"
	
*-------------------------------------------------------------------------------

cd "$main"
pwd

ssc install reghdfe, replace
ssc install ftools, replace
ssc install ivreghdfe, replace
ssc install ivreg2, replace
ssc install ftools, replace
ssc install ranktest, replace
ssc install gtools, replace

*-------------------------------------------------------------------------------
dis "$S_DATE" " " "$S_TIME"
*-------------------------------------------------------------------------------
*-------------------------------------------------------------------------------


*** Generate and prepare/ clean datasets

* Generate dataset with industry (ind) and grouped industry (sector) using BHP
cap log close
log using "$log\00_BHP_data_$S_DATE", text replace
** Input:
* "$BHP19\bhp_7519_m06_wgen_v2.dta"
** Output:
* "$temp/wz.dta"
do "$prog\00_BHP_data"
clear
cap log close

* Prepare BeH
cap log close
log using "$log\01_BeH_data_$S_DATE", text replace
** Input:
* "$data\a015784_beh.dta"
** Output:
* "$temp\BeH_large.dta"
do "$prog\01_BeH_data"
clear
cap log close

* Impute wages
cap log close
log using "$log\02_master_wage_imp_$S_DATE", text replace
** impute wages
** Input:
* "$temp\BeH_large.dta"
** Output:
* "$temp\imp_wage.dta"
* "$temp\BeH.dta"
** Note:
* overwrites "$temp\BeH_large.dta" from 00_BeH_data
* replaces tentgelt (daily wage) by imputed daily wage
* original tentgelt is saved in wage_imp
do "$imp\master_wage_imp.do"
clear
cap log close

* Identify movers and sample selection [drop (marginal) part-time spells]
cap log close
log using "$log\03_identify_movers_BeH_$S_DATE", text replace
** Input:
* "$temp\BeH.dta"
** Output:
* "$data\cleaned_obs_all.dta"
do "$prog\03_identify_moves_BeH"
clear
cap log close

*** Generate different datasets (spanning subsets of years) using loop
** Step 1: Indentify moves
local i_list = ""
foreach tw in 3 7 {
forval y = 1997(1)2019 {
if `y'-(`tw'-1)/2 > 1997 & `y'+(`tw'-1)/2 < 2019 {
local addto = `y'*10 + `tw'
local i_list "`i_list' `addto'"
}
}
}
dis "`i_list'"
foreach i in 3 `i_list' { 
	global ver = `i'  
	dis as text "ver = $ver"
	* Identify moves in BeH
	cap log close
	log using "$log\04_identify_moves_BeH_sample_selection_ver_`i'_$S_DATE", text replace
	** Input:
	* "$data\cleaned_obs_all.dta"
	** Output:
	* "$data\cleaned_obs_ver$ver.dta"
	* "$data/cleaned_obs_reconfig_hockey_ver$ver.dta"
	* "$data/movers_ver$ver.dta"
	* "$data/workers_currentid_year_ver$ver.dta"
	do "Set globals.do"
	do "$prog\04_identify_moves_BeH_sample_selection"
	clear
	cap log close
}
** Step 2: Generate hockey sticks and generate datasets for Matlab 
foreach i in 3 `i_list' { 
	global ver = `i'  
	dis as text "ver = $ver"
	do "Set globals.do"	
	* Generate hockey sticks
	cap log close
	log using "$log\05_mover_inputs_BeH_ver_`i'_$S_DATE", text replace
	** Input:
	* "$data/cleaned_obs_reconfig_hockey_ver$ver.dta"
	** Output:
	* "$data/move_rates_growth_raw_ver$ver.dta"
	* "$data/estab_eeenrates_byyr_ver$ver.dta"
	* "$graph/hockey_stick_cut_ver$ver.pdf"
	* "$graph/hockey_stick_ver$ver.pdf"
	do "$prog\05_mover_inputs_BeH"
	clear
	cap log close
	* Generate datasets for Matlab (to find connected sets)
	cap log close
	log using "$log\06_str_conn_input_BeH_ver_`i'_$S_DATE", text replace
	** Input:
	* "$data/movers_ver$ver.dta"
	* "$data/estab_eeenrates_byyr_ver$ver.dta"
	* "$data/estab_matrix_ids_ver$ver.dta"
	** Output:
	* "$data/str_conn_input_ver$ver.dta"
	* "$data/estab_matrix_ids_ver$ver.dta" is used within 04_
	* "$data/estab_matrix_ids_orig_ver$ver.dta"
	* "$data/restricted_senders_ver$ver.dta"
	* "$matlab\sconn_idfy\flows_ver$ver.csv"
	do "$prog\06_str_conn_input_BeH"
	clear
	cap log close
}

stop



********************************************************************************
********************************************************************************
********************************************************************************
*																			   *
* Run the following file in Matlab to find connected sets: sconn_idfy2.m       *
* After running Matlab, continue here:										   *
*																			   *
********************************************************************************
********************************************************************************
********************************************************************************



*** Generate files to estimate (main) Sorkin model
local i_list = ""
foreach tw in 3 7 {
forval y = 1997(1)2019 {
if `y'-(`tw'-1)/2 > 1997 & `y'+(`tw'-1)/2 < 2019 {
local addto = `y'*10 + `tw'
local i_list "`i_list' `addto'"
}
}
}
dis "`i_list'"
foreach i in 3 `i_list'  {
	global ver = `i'  
	dis as text "ver = $ver"
	do "Set globals.do"
	capture log close
	log using "$log/07_model_inputs_BeH_ver$ver_$S_DATE", text replace
	** Input:
	* "$matlab\sconn_idfy\sconn_idfy_ver$ver.csv"
	* "$data/estab_matrix_ids_orig_ver$ver.dta"
	* "$data/str_conn_input_ver$ver.dta"
	* "$data/estab_eeenrates_byyr_ver$ver.dta"
	* "$data\workers_currentid_year_ver$ver.dta"
	* "$temp/wz.dta"
	** Output:
	* "$data/estab_matrix_ids_ver$ver.dta"
	* "$data/fo_new_ver$ver.dta"
	* "$data/model_input_new_ver$ver.dta"
	* "$matlab\data_new\connected_moves_ver$ver.csv"
	* "$matlab\data_new\params_ver$ver.csv"
	do "$prog\07_model_inputs_BeH"
	clear
	cap log close

}

stop



********************************************************************************
********************************************************************************
********************************************************************************
*																			   *
* Run the following file in Matlab: estimate_model_v7.m 					   *
* After running Matlab, continue here:										   *
*																			   *
********************************************************************************
********************************************************************************
********************************************************************************



*** Generate alternative connected sets based on employer-skill combinations
local i_list = ""
foreach tw in 3 {
forval y = 1997(1)2019 {
if `y'-(`tw'-1)/2 > 1997 & `y'+(`tw'-1)/2 < 2019 {
local addto = `y'*10 + `tw'
local i_list "`i_list' `addto'"
}
}
}
dis "`i_list'"
foreach i in 3 `i_list' { 
	global ver = `i'  
	dis as text "ver = $ver"
	do "Set globals.do"	
	* Generate datasets for Matlab
	cap log close
	log using "$log\08_str_conn_input_BeH_skill_ver_`i'_$S_DATE", text replace
	** Input:
	* "$data/movers_ver$ver.dta"
	* "$data/estab_eeenrates_byyr_ver$ver.dta"
	* "$data/estab_matrix_ids_ver$ver.dta"
	** Output:
	* "$data/str_conn_input_skill_ver$ver.dta"
	* "$data/estab_matrix_ids_skill_ver$ver.dta" is used within 04_
	* "$data/estab_matrix_ids_orig_skill_ver$ver.dta"
	* "$data/restricted_senders_skill_ver$ver.dta"
	* "$matlab\sconn_idfy\flows_skill_ver$ver.csv"
	do "$prog\08_str_conn_input_BeH_skill"
	clear
	cap log close
}

stop



********************************************************************************
********************************************************************************
********************************************************************************
*																			   *
* Run the following file in Matlab to find connected sets: sconn_idfy2_skill.m *
* After running Matlab, continue here:										   *
*																			   *
********************************************************************************
********************************************************************************
********************************************************************************



*** Generate files to estimate Sorkin model for employer-skill combinations
local i_list = ""
foreach tw in 3 {
forval y = 1997(1)2019 {
if `y'-(`tw'-1)/2 > 1997 & `y'+(`tw'-1)/2 < 2019 {
local addto = `y'*10 + `tw'
local i_list "`i_list' `addto'"
}
}
}
dis "`i_list'"
foreach i in 3 `i_list' { 
	global ver = `i'  
	dis as text "ver = $ver"
	do "Set globals.do"
	capture log close
	log using "$log/09_model_inputs_BeH_skill_ver${ver}_$S_DATE", text replace
	** Input:
	* "$matlab\sconn_idfy\sconn_idfy_ver$ver.csv"
	* "$data/estab_matrix_ids_orig_ver$ver.dta"
	* "$data/str_conn_input_ver$ver.dta"
	* "$data/estab_eeenrates_byyr_ver$ver.dta"
	* "$data\workers_currentid_year_ver$ver.dta"
	* "$temp/wz.dta"
	** Output:
	* "$data/estab_matrix_ids_ver$ver.dta"
	* "$data/fo_new_ver$ver.dta"
	* "$data/model_input_new_ver$ver.dta"
	* "$matlab\data_new\connected_moves_ver$ver.csv"
	* "$matlab\data_new\params_ver$ver.csv"
	do "$prog\09_model_inputs_BeH_skill"
	clear
	cap log close
}

stop



********************************************************************************
********************************************************************************
********************************************************************************
*																			   *
* Run the following file in Matlab: estimate_model_skill_v7.m				   *
* After running Matlab, continue here:										   *
*																			   *
********************************************************************************
********************************************************************************
********************************************************************************



*** Estimating AKM effects
local i_list = ""
foreach tw in 3 7 {
forval y = 1997(1)2019 {
if `y'-(`tw'-1)/2 > 1997 & `y'+(`tw'-1)/2 < 2019 {
local addto = `y'*10 + `tw'
local i_list "`i_list' `addto'"
}
}
}
dis "`i_list'"
foreach i in `i_list' 3 {
	global ver = `i'  
	dis as text "ver = $ver"
	do "Set globals.do"	
	capture log close
	log using "$log/10_AKM_ver${ver}_$S_DATE", text replace
	do "$prog\10_AKM.do"
	clear
	cap log close
}

*** Estimating AKM effects for employer-skill combinations 
local i_list = ""
foreach tw in 3 {
forval y = 1997(1)2019 {
if `y'-(`tw'-1)/2 > 1997 & `y'+(`tw'-1)/2 < 2019 {
local addto = `y'*10 + `tw'
local i_list "`i_list' `addto'"
}
}
}
dis "`i_list'"
foreach i in `i_list' {
	global ver = `i'  
	dis as text "ver = $ver"
	do "Set globals.do"
	capture log close
	log using "$log/11_AKM_skill_ver${ver}_$S_DATE", text replace
	do "$prog\11_AKM_skill.do"
	clear
	cap log close
}

*** Produce statistics about values and AKM effects etc.
local i_list = ""
foreach tw in 3 7 {
forval y = 1997(1)2019 {
if `y'-(`tw'-1)/2 > 1997 & `y'+(`tw'-1)/2 < 2019 {
local addto = `y'*10 + `tw'
local i_list "`i_list' `addto'"
}
}
}
dis "`i_list'"
foreach i in `i_list' 3 {  
	global ver = `i'  
	dis as text "ver = $ver"
	do "Set globals.do"
	capture log close
	log using "$log/12_analyze_values_BeH_ver${ver}_${S_DATE}", text replace
	** Input:
	* "$matlab\data_new\exp_V_ver$ver.csv"
	* "$data/workers_currentid_year_ver$ver.dta"
	* "$AKM/akm_estab_ver$ver.dta"
	* "$temp/wz.dta"
	** Output:
	* "$data/values_new_ver$ver.dta"
	* "$data/workers_betnr_ver$ver.dta"
	* "$data/firmfe_value_new_ver$ver"
	* "$results/V_fe_p5_2_99_ver$ver.csv"
	* "$graph/V_fe_p5_2_99_ver$ver.pdf"
	* "$results/the_one_ver$ver.dta"
	* "$results/firmfe_value_sector_ver$ver.csv"
	* "$graph/sector_map_ver$ver.pdf"
	* "$graph/sector_map_wo_78_ver$ver.pdf"
	* "$graph/sector_map_wo_78_new_ver$ver.pdf"
	do "$prog\12_analyze_values_BeH"
	clear
	cap log close
}

*** Produce statistics about values and AKM effects etc. for employer-skill combinations
local i_list = "19993 20023 20053 20083 20113 20143 20173" 
foreach i in `i_list' {
 	global ver = `i'  
	dis as text "ver = $ver"
	do "Set globals.do"
	capture log close
	log using "$log/13_analyze_values_BeH_skill_ver${ver}_${S_DATE}", text replace
	** Input:
	* "$matlab\data_new\exp_V_ver$ver.csv"
	* "$data/workers_currentid_year_ver$ver.dta"
	* "$AKM/akm_estab_ver$ver.dta"
	* "$temp/wz.dta"
	** Output:
	* "$data/values_new_ver$ver.dta"
	* "$data/workers_betnr_ver$ver.dta"
	* "$data/firmfe_value_new_ver$ver"
	* "$results/V_fe_p5_2_99_ver$ver.csv"
	* "$graph/V_fe_p5_2_99_ver$ver.pdf"
	* "$results/the_one_ver$ver.dta"
	* "$results/firmfe_value_sector_ver$ver.csv"
	* "$graph/sector_map_ver$ver.pdf"
	* "$graph/sector_map_wo_78_ver$ver.pdf"
	* "$graph/sector_map_wo_78_new_ver$ver.pdf"
	do "$prog\13_analyze_values_BeH_skill" 
	clear
	cap log close
}


*******************************************************************************************
*******************************************************************************************
do "prog/14_split_sample_master.do" // This program creates split sample estimates of values and AKM effects
*******************************************************************************************
*******************************************************************************************



*******************************************************************************************
*******************************************************************************************
*** Generate Rents and CD Estimates for Analysis
do "prog/15_01_split_sample_check_3stack.do" // Fig 1; Fig 2; Table F1
do "prog/15_02_split_sample_check_3stack_grand.do" 
do "prog/15_03_split_sample_check_7stack.do" 
do "prog/15_04_split_sample_check_full.do"   
do "prog/15_05_split_sample_check_3stack_skill.do" 
*** Generate Statistics 
do "prog/16_01_3stack_summary_table.do" // Statistics for Table 1; Table C1; Table E3 
do "prog/16_02_7stack_summary_table.do" // Statistics for Table E3  
do "prog/16_03_variance_decomp.do" // Statistics for Table E3

*******************************************************************************************
*******************************************************************************************
*****************************************************

clear
cap close
